# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from urllib.parse import urljoin
from selenium import webdriver
from items import zufang_lianjiaItemLoader
from items import zufang_lianjiaItem
import re


class ZufangLianjiaSpider(scrapy.Spider):
    name = 'zufang_lianjia'
    allowed_domains = ['bj.lianjia.com/zufang/']
    start_urls = ['https://bj.lianjia.com/zufang/']

    def __init__(self):
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference('permissions.default.image', 2)

        self.driver = webdriver.Firefox(firefox_profile=firefox_profile,
                                        executable_path="/home/demeen/Desktop/source/geckodriver")
        super(ZufangLianjiaSpider, self).__init__()
        from scrapy.xlib.pydispatch import dispatcher
        from scrapy import signals
        dispatcher.connect(self.spider_closed, signals.spider_closed)

    def spider_closed(self, spider):
        print("spider close")
        self.driver.quit()

    def parse(self, response):
        hose_list = response.css("#house-lst")
        hose_list = hose_list.css(".info-panel h2")
        hose_list = hose_list.css("a::attr(href)").extract()
        for house_url in hose_list:
            yield Request(url=urljoin(response.url, house_url), callback=self.parse_house, dont_filter=True)
        next_page = response.css(".page-box.house-lst-page-box a:last-child")
        next_page = next_page.css("a::attr(href)").extract_first()
        if next_page:
            next_url = urljoin(response.url, next_page)
            yield Request(url=next_url, callback=self.parse, dont_filter=True)

    def parse_house(self, response):
        item_loader = zufang_lianjiaItemLoader(item=zufang_lianjiaItem(), response=response)
        item_loader.add_value("id", str(response.url).split('/')[-1])

        # analyze the house page
        title = response.css(".content-wrapper .title-wrapper .title .main::text").extract_first()
        item_loader.add_value("title", title)
        # money and base
        base_plane = response.css(".content.zf-content")
        price = base_plane.css("span.total::text").extract_first()
        item_loader.add_value("price", price)
        fitment = base_plane.css("span.tips::text").extract_first()
        item_loader.add_value("fitment", fitment)

        zf_room = base_plane.css(".zf-room p::text").extract()  # we only need [0,4] details
        item_loader.add_value("squire", zf_room[0])
        item_loader.add_value("room_status", zf_room[1])
        item_loader.add_value("floor", zf_room[2])
        item_loader.add_value("face_to", zf_room[3])
        item_loader.add_value("subway", zf_room[4])
        # detail direction
        base_direction = response.css("#introduction .content li::text").extract()
        real_direction = []
        for direction in base_direction:
            direction = str(direction).strip()
            if direction:
                real_direction.append(direction)
        item_loader.add_value("rant_way", real_direction[0])
        item_loader.add_value("pay_way", real_direction[1])
        item_loader.add_value("now_status", real_direction[2])
        item_loader.add_value("hot_way", real_direction[3])
        # it's long ,so we should make it after all analyze we have
        long_description = response.css(".featureContent .text").extract()
        # what we have in the room
        departments = response.css(".zf-tag li.tags").extract()
        department = ""
        for depart in departments:
            depart = re.sub(re.compile(r'<[^>]+>', re.S), '', depart).strip()
            department = department + depart + ","
        item_loader.add_value("tag", department)
        # time by analyze
        panel = response.css("#record .panel")
        visit_in_seven = panel.css(".count").extract_first()
        visit_in_seven = re.sub(re.compile(r'<[^>]+>', re.S), '', visit_in_seven).strip()
        item_loader.add_value("visit_in_seven", visit_in_seven)
        visit_in_thirty = panel.css("span::text").extract_first()
        item_loader.add_value("visit_in_thirty", visit_in_thirty)
        # make long_description can save
        long_des = ""
        for detail in long_description:
            detail = re.sub(re.compile(r'<[^>]+>', re.S), '', detail).strip()
            long_des = long_des + "@" + detail
        item_loader.add_value("long_description", long_des)
        item = item_loader.load_item()
        yield item
